from bs4 import BeautifulSoup
import requests
import time
import pymongo
import re

client = pymongo.MongoClient('localhost', 27017)
ceshi = client['ceshi']
url_list = ceshi['url_list']
item_info = ceshi['item_info']


# 在最左边是在python 中对象的名称，后面的是在数据库中的名称
# spider 1
def get_links_from(channel, pages, who_sells=0):
    # td.t 没有这个就终止
    list_view = '{}{}/pn{}/'.format(channel, str(who_sells), str(pages))
    wb_data = requests.get(list_view)
    time.sleep(1)
    soup = BeautifulSoup(wb_data.text, 'lxml')
    if soup.find('td', 't'):
        for link in soup.select('td.t a.t'):
            item_link = link.get('href').split('?')[0]
            url_list.insert_one({'url': item_link})
            print(item_link)
            # return urls
    else:
        # It's the last page !
        pass
def clean_zh_text(text):
    comp = re.compile('[^A-Z^a-z^0-9^\u4e00-\u9fa5]')
    return comp.sub('', text)

def clean_en_text(text):
    comp = re.compile('[^A-Z^a-z^0-9^ ]')
    return comp.sub('', text)


# spider 2
def get_item_info(url):
    wb_data = requests.get(url)
    soup = BeautifulSoup(wb_data.text, 'lxml')
    no_longer_exist = '404-58同城' in soup.select('head > title')[0].text
    if no_longer_exist:
        pass
    else:
        title = clean_zh_text(soup.select('div.detail-title > h1')[0].get_text())
        # print(title)
        # price = clean_en_text(soup.select(' div.infocard__container__item__main')[0].get_text())
        price = re.findall(r"\d+\.?\d*",soup.select(' div.infocard__container__item__main')[0].get_text())[0]
        # print(price)
        date =re.search(r"(\d{4}-\d{1,2}-\d{1,2})",soup.select('div.detail-title__info > div:nth-child(1)')[0].get_text())[0]
        # print(date)
        area = list(soup.select('div.infocard__container__item__main > a')[0].stripped_strings)[0] if list(soup.select('div.infocard__container__item__main > a')[0].stripped_strings)[0] else None
        # print(area)
        # area = list(soup.select('.c_25d a')[0].stripped_strings) if soup.find_all('span', 'c_25d') else None
        item_info.insert_one({'title': title, 'price': price, 'date': date, 'area': area, 'url': url})
        print({'title': title, 'price': price, 'date': date, 'area': area, 'url': url})


# get_links_from('http://bj.58.com/shouji/',2)

# get_item_info('https://bj.58.com/shouji/42560195694629x.shtml')

# url = 'https://bj.58.com/shouji/24605954621114x.shtml'
# wb_data = requests.get(url)
# soup = BeautifulSoup(wb_data.text,'lxml')
# print(soup.prettify())

db_urls = [item['url'] for item in url_list.find()]  # 用列表解析式装入所有要爬取的链接
index_urls = [item['url'] for item in item_info.find()] # 所引出详情信息数据库中所有的现存的 url 字段
x = set(db_urls)    # 转换成集合的数据结构
y = set(index_urls)
rest_of_urls = x-y            # 相减
for x in rest_of_urls:
    get_item_info(x)
    time.sleep(2)
print('Done')